import time
import requests
from lxml import etree

url = r'https://www.boxofficemojo.com/quarter/q2/2020/?grossesOption=totalGrosses' #第二季度上映电影的网址
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'} #爬虫头部
rsp = requests.get(url, headers=headers) #读取网页
text = rsp.text #获取网页源码
html = etree.HTML(text)
movie_relative_urls =  html.xpath(r'//td[@class="a-text-left mojo-field-type-release mojo-cell-wide"]/a/@href') #获取每部电影的信息页面的相对地址
movie_urls = [r'https://www.boxofficemojo.com'+u for u in movie_relative_urls] #把每部电影的相对地址换成绝对地址
genres_dict = {} #用于保存信息的字典

def spider(url): #这个函数主要用于读取每部电影页面中的电影分级信息
    rsp = requests.get(url, headers=headers) #读取每部电影的网页
    text = rsp.text #获取页面代码
    html = etree.HTML(text)
    genre = html.xpath(r'//div/span[text()="Genres"]/following-sibling::span[1]/text()')[0] #读取电影分级信息
    title = html.xpath(r'//div/h1/text()')[0] #读取电影名称
    genres_dict[title] = genre #把每部电影的名称和分级信息存入字典
    
normal_start = time.time() #程序开始时间
for u in movie_urls:
    spider(u)
normal_end = time.time() #程序结束时间
normal_elapse = normal_end - normal_start #程序运行时间
print('The normal procedure costs %s seconds' % normal_elapse)
